Setup¶
Path to training data on drive¶
In [ ]:
path_folder = "../splits_final_deblurred/"
Imports¶
In [ ]:
from __future__ import print_function
import tensorflow as tf
from tensorflow import keras
from keras.datasets import mnist
#import tensorflow.keras.backend as K
from matplotlib import pyplot as plt
from keras.models import Sequential
from keras.layers import Dense, Flatten, Activation, concatenate, BatchNormalization
import matplotlib.pyplot as plt
import numpy as np
from keras.activations import softmax
from keras.layers import concatenate
from keras.layers import UpSampling2D
from keras.layers import Dropout
from keras.layers import Conv2D, MaxPooling2D
from keras.layers import Input
from keras.models import Model
import sys
import cv2
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Softmax
import os
from PIL import Image
Helper Functions¶
In [ ]:
import cv2
import numpy as np
import matplotlib.pyplot as plt
num_classes = 5
class_names = ["background", "rigid_plastic", "cardboard", "metal", "soft_plastic"]
class_colors = [[0,0,0], [0,0,255], [0,255,0], [255,0,0] ,[125,0,125]]
def Make_Overlapping_Plot2(image, label=None, pred=None):
'''
Makes one/two/three plots, first is the image, the others are segmentation/predicitons combined with image where background is unchanged, all other classes
have a color.
\n RED: Rigid Plastic
\n GREEN: Cardboard
\n BLUE: Metal
\n PURPLE: Soft plastic.
'''
_, ax1 = plt.subplots()
ax1.set_title("Image")
ax1.imshow(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
if label is not None:
red = np.where(label[:,:,1] == 1)
green = np.where(label[:,:,2] == 1)
blue = np.where(label[:,:,3] == 1)
purple = np.where(label[:,:,4] == 1)
image2 = np.copy(image)
image2[red] = class_colors[1]
image2[green] = class_colors[2]
image2[blue] = class_colors[3]
image2[purple] = class_colors[4]
_, ax2 =plt.subplots()
ax2.set_title("Overlapping Labels")
ax2.imshow(cv2.cvtColor(image2, cv2.COLOR_BGR2RGB))
if pred is not None:
image3 = np.copy(image)
red = np.where(pred[:,:,1] >= 0.5)
green = np.where(pred[:,:,2] >= 0.5)
blue = np.where(pred[:,:,3] >= 0.5)
purple = np.where(pred[:,:,4] >= 0.5)
image3[red] = class_colors[1]
image3[green] = class_colors[2]
image3[blue] = class_colors[3]
image3[purple] = class_colors[4]
_, ax3 =plt.subplots()
ax3.set_title("predictions")
ax3.imshow(cv2.cvtColor(image3, cv2.COLOR_BGR2RGB))
plt.show()
return 0
def create_y_labels(loaded_labels, y_labels):
if loaded_labels.ndim == 3:
for i in range(0,y_labels.shape[0]):
background = np.where(loaded_labels[i,:,:] == 0)
red = np.where(loaded_labels[i,:,:] == 1)
green = np.where(loaded_labels[i,:,:] == 2)
blue = np.where(loaded_labels[i,:,:] == 3)
purple = np.where(loaded_labels[i,:,:] == 4)
y_labels[i,:,:,0][background] = 1 #Background
y_labels[i,:,:,1][red] = 1 #Rigid Plast
y_labels[i,:,:,2][green] = 1 #Cardboard
y_labels[i,:,:,3][blue] = 1 #Metal
y_labels[i,:,:,4][purple] = 1 #SoftPlast
else:
for i in range(0,y_labels.shape[0]):
background = np.where(loaded_labels[i,:,:,0] == 0)
red = np.where(loaded_labels[i,:,:,0] == 1)
green = np.where(loaded_labels[i,:,:,0] == 2)
blue = np.where(loaded_labels[i,:,:,0] == 3)
purple = np.where(loaded_labels[i,:,:,0] == 4)
y_labels[i,:,:,0][background] = 1 #Background
y_labels[i,:,:,1][red] = 1 #Rigid Plast
y_labels[i,:,:,2][green] = 1 #Cardboard
y_labels[i,:,:,3][blue] = 1 #Metal
y_labels[i,:,:,4][purple] = 1 #SoftPlast
return y_labels
VGG16-UNet Experiment¶
Model¶
Loss function
In [ ]:
import keras.backend as K
def dice_coef(y_true, y_pred):
y_true = tf.reshape(tf.cast(y_true, tf.float64), [-1])
y_pred = tf.reshape(tf.cast(y_pred, tf.float64), [-1])
intersection = K.sum(y_true * y_pred)
union = K.sum(y_true) + K.sum(y_pred)
epsilon = 1e-4
return (2*intersection + epsilon)/(union+epsilon)
def dice_loss_function(y_true, y_pred):
coef = 0.
for i in range(5):
coef += dice_coef(y_true[:,:,:,i], y_pred[:,:,:,i])
return 1-coef/5.
Similar to the model described in https://qims.amegroups.org/article/view/91409/pdf , page 3143, figure 3(b)
In [ ]:
from keras.layers import concatenate, BatchNormalization
from keras.layers import UpSampling2D
from keras.activations import softmax
from keras.applications.vgg16 import preprocess_input
from tensorflow.keras.losses import categorical_crossentropy
input_dim = (224, 224)
# See last layer of network
def softMaxAxis3(x):
return softmax(x,axis=3)
def my_conv(x,filters,kernel_size=3,padding='same',kernel_initializer='he_normal'):
x = Conv2D(filters, kernel_size, padding=padding, kernel_initializer=kernel_initializer, strides=1)(x)
#x = BatchNormalization()(x)
x = Activation('relu')(x)
return x
def apply_tensor_to_layers(x, layers):
for l in layers:
x = l(x)
return x
vgg16 = keras.applications.VGG16(weights="imagenet",input_shape=(*input_dim, 3), include_top=False)
inputs = keras.Input(shape=(*input_dim, 3))
x = preprocess_input(inputs) #shifts from RBG to BGR and zero centers pixels, BGR mean values = [-103.93900299, -116.77899933, -123.68000031]
x = vgg16.layers[1](x, training=False)
# create vgg16 encoder while remembering connections for concatenation
con1 = apply_tensor_to_layers(x, vgg16.layers[2:3])
con2 = apply_tensor_to_layers(con1, vgg16.layers[3:6])
con3 = apply_tensor_to_layers(con2, vgg16.layers[6:10])
con4 = apply_tensor_to_layers(con3, vgg16.layers[10:14])
con5 = apply_tensor_to_layers(con4, vgg16.layers[14:18])
encoder = apply_tensor_to_layers(con5, vgg16.layers[18:19])
print(con5)
# decoder
up1 = my_conv(encoder, 1024)
up1 = my_conv(up1, 512)
up1 = UpSampling2D((2,2))(up1)
concat5 = concatenate([up1, con5] ,axis=3)
up2 = my_conv(concat5, filters=512)
up2 = my_conv(up2, filters=512)
up2 = UpSampling2D((2,2))(up2)
concat4 = concatenate([up2, con4], axis=3)
up3 = my_conv(concat4, filters=256)
up3 = my_conv(up3, filters=256)
up3 = UpSampling2D((2,2))(up3)
concat3 = concatenate([up3, con3], axis=3)
up4 = my_conv(concat3, filters=128)
up4 = my_conv(up4, filters=128)
up4 = UpSampling2D((2,2))(up4)
concat2 = concatenate([up4, con2], axis=3)
up5 = my_conv(concat2, filters=64)
up5 = my_conv(up5, filters=64)
up5 = UpSampling2D((2,2))(up5)
concat1 = concatenate([up5, con1], axis=3)
decoder = my_conv(concat1, filters=16)
decoder = Conv2D(num_classes, 1, activation = softMaxAxis3)(decoder)
# Model
model = Model(inputs, decoder)
#set vgg16 layers to not trainable
for i in range(19):
model.layers[i].trainable = False
#, weight_decay=1e-6, momentum=0.9, nesterov=True
model.compile(optimizer = keras.optimizers.SGD(learning_rate = 0.01, clipnorm = 1), loss = tf.keras.losses.CategoricalCrossentropy(), metrics=['accuracy'])
model.summary()
2023-12-07 16:25:24.309634: I metal_plugin/src/device/metal_device.cc:1154] Metal device set to: Apple M1 2023-12-07 16:25:24.309657: I metal_plugin/src/device/metal_device.cc:296] systemMemory: 16.00 GB 2023-12-07 16:25:24.309661: I metal_plugin/src/device/metal_device.cc:313] maxCacheSize: 5.33 GB 2023-12-07 16:25:24.309690: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:306] Could not identify NUMA node of platform GPU ID 0, defaulting to 0. Your kernel may not have been built with NUMA support. 2023-12-07 16:25:24.309706: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:272] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 0 MB memory) -> physical PluggableDevice (device: 0, name: METAL, pci bus id: <undefined>) WARNING:absl:At this time, the v2.11+ optimizer `tf.keras.optimizers.SGD` runs slowly on M1/M2 Macs, please use the legacy Keras optimizer instead, located at `tf.keras.optimizers.legacy.SGD`.
KerasTensor(type_spec=TensorSpec(shape=(None, 14, 14, 512), dtype=tf.float32, name=None), name='block5_conv3/Relu:0', description="created by layer 'block5_conv3'")
Model: "model"
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
input_2 (InputLayer) [(None, 224, 224, 3)] 0 []
tf.__operators__.getitem ( (None, 224, 224, 3) 0 ['input_2[0][0]']
SlicingOpLambda)
tf.nn.bias_add (TFOpLambda (None, 224, 224, 3) 0 ['tf.__operators__.getitem[0][
) 0]']
block1_conv1 (Conv2D) (None, 224, 224, 64) 1792 ['tf.nn.bias_add[0][0]']
block1_conv2 (Conv2D) (None, 224, 224, 64) 36928 ['block1_conv1[1][0]']
block1_pool (MaxPooling2D) (None, 112, 112, 64) 0 ['block1_conv2[1][0]']
block2_conv1 (Conv2D) (None, 112, 112, 128) 73856 ['block1_pool[1][0]']
block2_conv2 (Conv2D) (None, 112, 112, 128) 147584 ['block2_conv1[1][0]']
block2_pool (MaxPooling2D) (None, 56, 56, 128) 0 ['block2_conv2[1][0]']
Model: "model"
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
input_2 (InputLayer) [(None, 224, 224, 3)] 0 []
tf.__operators__.getitem ( (None, 224, 224, 3) 0 ['input_2[0][0]']
SlicingOpLambda)
tf.nn.bias_add (TFOpLambda (None, 224, 224, 3) 0 ['tf.__operators__.getitem[0][
) 0]']
block1_conv1 (Conv2D) (None, 224, 224, 64) 1792 ['tf.nn.bias_add[0][0]']
block1_conv2 (Conv2D) (None, 224, 224, 64) 36928 ['block1_conv1[1][0]']
block1_pool (MaxPooling2D) (None, 112, 112, 64) 0 ['block1_conv2[1][0]']
block2_conv1 (Conv2D) (None, 112, 112, 128) 73856 ['block1_pool[1][0]']
block2_conv2 (Conv2D) (None, 112, 112, 128) 147584 ['block2_conv1[1][0]']
block2_pool (MaxPooling2D) (None, 56, 56, 128) 0 ['block2_conv2[1][0]']
block3_conv1 (Conv2D) (None, 56, 56, 256) 295168 ['block2_pool[1][0]']
block3_conv2 (Conv2D) (None, 56, 56, 256) 590080 ['block3_conv1[1][0]']
block3_conv3 (Conv2D) (None, 56, 56, 256) 590080 ['block3_conv2[1][0]']
block3_pool (MaxPooling2D) (None, 28, 28, 256) 0 ['block3_conv3[1][0]']
block4_conv1 (Conv2D) (None, 28, 28, 512) 1180160 ['block3_pool[1][0]']
block4_conv2 (Conv2D) (None, 28, 28, 512) 2359808 ['block4_conv1[1][0]']
block4_conv3 (Conv2D) (None, 28, 28, 512) 2359808 ['block4_conv2[1][0]']
block4_pool (MaxPooling2D) (None, 14, 14, 512) 0 ['block4_conv3[1][0]']
block5_conv1 (Conv2D) (None, 14, 14, 512) 2359808 ['block4_pool[1][0]']
block5_conv2 (Conv2D) (None, 14, 14, 512) 2359808 ['block5_conv1[1][0]']
block5_conv3 (Conv2D) (None, 14, 14, 512) 2359808 ['block5_conv2[1][0]']
block5_pool (MaxPooling2D) (None, 7, 7, 512) 0 ['block5_conv3[1][0]']
conv2d (Conv2D) (None, 7, 7, 1024) 4719616 ['block5_pool[1][0]']
activation (Activation) (None, 7, 7, 1024) 0 ['conv2d[0][0]']
conv2d_1 (Conv2D) (None, 7, 7, 512) 4719104 ['activation[0][0]']
activation_1 (Activation) (None, 7, 7, 512) 0 ['conv2d_1[0][0]']
up_sampling2d (UpSampling2 (None, 14, 14, 512) 0 ['activation_1[0][0]']
D)
concatenate (Concatenate) (None, 14, 14, 1024) 0 ['up_sampling2d[0][0]',
'block5_conv3[1][0]']
conv2d_2 (Conv2D) (None, 14, 14, 512) 4719104 ['concatenate[0][0]']
activation_2 (Activation) (None, 14, 14, 512) 0 ['conv2d_2[0][0]']
conv2d_3 (Conv2D) (None, 14, 14, 512) 2359808 ['activation_2[0][0]']
activation_3 (Activation) (None, 14, 14, 512) 0 ['conv2d_3[0][0]']
up_sampling2d_1 (UpSamplin (None, 28, 28, 512) 0 ['activation_3[0][0]']
g2D)
concatenate_1 (Concatenate (None, 28, 28, 1024) 0 ['up_sampling2d_1[0][0]',
) 'block4_conv3[1][0]']
conv2d_4 (Conv2D) (None, 28, 28, 256) 2359552 ['concatenate_1[0][0]']
activation_4 (Activation) (None, 28, 28, 256) 0 ['conv2d_4[0][0]']
conv2d_5 (Conv2D) (None, 28, 28, 256) 590080 ['activation_4[0][0]']
activation_5 (Activation) (None, 28, 28, 256) 0 ['conv2d_5[0][0]']
up_sampling2d_2 (UpSamplin (None, 56, 56, 256) 0 ['activation_5[0][0]']
g2D)
concatenate_2 (Concatenate (None, 56, 56, 512) 0 ['up_sampling2d_2[0][0]',
) 'block3_conv3[1][0]']
conv2d_6 (Conv2D) (None, 56, 56, 128) 589952 ['concatenate_2[0][0]']
activation_6 (Activation) (None, 56, 56, 128) 0 ['conv2d_6[0][0]']
conv2d_7 (Conv2D) (None, 56, 56, 128) 147584 ['activation_6[0][0]']
activation_7 (Activation) (None, 56, 56, 128) 0 ['conv2d_7[0][0]']
up_sampling2d_3 (UpSamplin (None, 112, 112, 128) 0 ['activation_7[0][0]']
g2D)
concatenate_3 (Concatenate (None, 112, 112, 256) 0 ['up_sampling2d_3[0][0]',
) 'block2_conv2[1][0]']
conv2d_8 (Conv2D) (None, 112, 112, 64) 147520 ['concatenate_3[0][0]']
activation_8 (Activation) (None, 112, 112, 64) 0 ['conv2d_8[0][0]']
conv2d_9 (Conv2D) (None, 112, 112, 64) 36928 ['activation_8[0][0]']
activation_9 (Activation) (None, 112, 112, 64) 0 ['conv2d_9[0][0]']
up_sampling2d_4 (UpSamplin (None, 224, 224, 64) 0 ['activation_9[0][0]']
g2D)
concatenate_4 (Concatenate (None, 224, 224, 128) 0 ['up_sampling2d_4[0][0]',
) 'block1_conv2[1][0]']
conv2d_10 (Conv2D) (None, 224, 224, 16) 18448 ['concatenate_4[0][0]']
activation_10 (Activation) (None, 224, 224, 16) 0 ['conv2d_10[0][0]']
conv2d_11 (Conv2D) (None, 224, 224, 5) 85 ['activation_10[0][0]']
==================================================================================================
Total params: 35122469 (133.98 MB)
Trainable params: 22767589 (86.85 MB)
Non-trainable params: 12354880 (47.13 MB)
__________________________________________________________________________________________________
plot model
Data Generator¶
Class implementation¶
In [ ]:
import numpy as np
import keras
import os
from PIL import Image
import cv2
class DataGenerator(keras.utils.Sequence):
def __init__(self, base_path, data, batch_size=32, dim=(240, 360), n_channels=3, n_classes=5, shuffle=True):
self.dim = dim
self.batch_size = batch_size
self.data = data
self.n_channels = n_channels
self.n_classes = n_classes
self.shuffle = shuffle
self.base_path = base_path
self.indexes = np.array(1)
self.on_epoch_end()
def __len__(self):
return int(np.floor(len(self.data) / self.batch_size))
def __getitem__(self, index):
if self.shuffle:
indexes = np.random.default_rng().choice(len(self.data), self.batch_size, replace=False)
i = np.random.randint(len(self.data)-self.batch_size)
else:
indexes = self.indexes[index * self.batch_size:(index + 1) * self.batch_size]
data_temp = [self.data[i] for i in indexes]
x, y = self.__data_generation(data_temp)
return x, y
def __getitem__(self, index):
indexes = self.indexes[index * self.batch_size:(index + 1) * self.batch_size]
data_temp = [self.data[i] for i in indexes]
x, y = self.__data_generation(data_temp)
return x, y
def on_epoch_end(self):
self.indexes = np.arange(len(self.data))
if self.shuffle:
np.random.shuffle(self.indexes)
def __data_generation(self, data):
x = np.empty((self.batch_size, self.dim[0], self.dim[1], self.n_channels), dtype=np.float32)
y = np.empty((self.batch_size, self.dim[0], self.dim[1], self.n_channels), dtype=np.float32)
for i, file_name in enumerate(data):
image_path = os.path.join(self.base_path, 'data', file_name)
seg_path = os.path.join(self.base_path, 'sem_seg', file_name)
try:
img_data = cv2.cvtColor(cv2.imread(image_path), cv2.COLOR_BGR2RGB)
index_for_square_cropping = np.random.randint(840)
img_data = img_data[:, index_for_square_cropping : index_for_square_cropping+1080, :]
img_data = cv2.resize(img_data, (self.dim[1], self.dim[0]), interpolation = cv2.INTER_NEAREST)
x[i] = np.array(img_data)
img_seg = cv2.imread(seg_path)
img_seg = img_seg[:, index_for_square_cropping : index_for_square_cropping+1080, :]
img_seg = cv2.resize(img_seg, (self.dim[1], self.dim[0]), interpolation = cv2.INTER_NEAREST)
y[i] = np.array(img_seg)
except Exception as e:
print(f"Error loading {image_path}: {str(e)}")
x = np.array(x, dtype=np.float32)
y = create_y_labels(y, np.zeros((self.batch_size, self.dim[0], self.dim[1], self.n_classes), dtype=np.float32))
return x, y
Load image file information including paths¶
In [ ]:
import json
import cv2
import numpy as np
def load_and_extract_ids(folder_path, data_type):
file_path = f"{folder_path}/{data_type}/labels.json"
with open(file_path) as file:
data_loader = json.load(file)
images = data_loader["images"]
print(len(images))
image_paths = []
for i in range(0,len(images)): #len(images)
image_paths.append(images[i]['file_name'])
return image_paths
train_ids = load_and_extract_ids(path_folder, "train")
validation_ids = load_and_extract_ids(path_folder, "val")
test_ids = load_and_extract_ids(path_folder, "test")
3002 572 929
Create Data Generator Instance¶
In [ ]:
params = {'dim': input_dim,
'batch_size': 32,
'n_classes': 5,
'n_channels': 3,
'shuffle': True}
partition = {
'train': train_ids,
'val': validation_ids,
'test': test_ids
}
training_generator = DataGenerator(base_path = path_folder + 'train/', data = partition['train'], **params)
validation_generator = DataGenerator(base_path = path_folder + 'val/', data = partition['val'], **params)
test_generator = DataGenerator(base_path = path_folder + 'test/', data = partition['test'], **params)
Training¶
Learning rate and fitting
In [ ]:
def our_learning_rate(epoch, learning_rate):
if epoch == 0:
return 0.01
return learning_rate * tf.math.exp(-0.10)
callback = keras.callbacks.LearningRateScheduler(our_learning_rate)
history = model.fit( training_generator, validation_data=validation_generator, epochs=20, callbacks = [callback], verbose=1)
Epoch 1/20
2023-12-07 16:25:27.616771: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:117] Plugin optimizer for device_type GPU is enabled. 2023-12-07 16:25:27.732356: E tensorflow/core/grappler/optimizers/meta_optimizer.cc:961] model_pruner failed: INVALID_ARGUMENT: Graph does not contain terminal node SGD/AssignVariableOp.
93/93 [==============================] - 243s 3s/step - loss: 10.7604 - accuracy: 0.8016 - val_loss: 1.1336 - val_accuracy: 0.8158 - lr: 0.0100 Epoch 2/20 93/93 [==============================] - 234s 2s/step - loss: 1.0368 - accuracy: 0.8382 - val_loss: 0.9307 - val_accuracy: 0.8109 - lr: 0.0090 Epoch 3/20 93/93 [==============================] - 231s 2s/step - loss: 0.8820 - accuracy: 0.8374 - val_loss: 0.8366 - val_accuracy: 0.8142 - lr: 0.0082 Epoch 4/20 93/93 [==============================] - 219s 2s/step - loss: 0.7990 - accuracy: 0.8374 - val_loss: 0.7797 - val_accuracy: 0.8161 - lr: 0.0074 Epoch 5/20 93/93 [==============================] - 219s 2s/step - loss: 0.7473 - accuracy: 0.8363 - val_loss: 0.7582 - val_accuracy: 0.8125 - lr: 0.0067 Epoch 6/20 93/93 [==============================] - 219s 2s/step - loss: 0.7125 - accuracy: 0.8359 - val_loss: 0.7826 - val_accuracy: 0.8136 - lr: 0.0061 Epoch 7/20 93/93 [==============================] - 215s 2s/step - loss: 0.6911 - accuracy: 0.8370 - val_loss: 0.7239 - val_accuracy: 0.8157 - lr: 0.0055 Epoch 8/20 93/93 [==============================] - 217s 2s/step - loss: 0.6762 - accuracy: 0.8379 - val_loss: 0.7065 - val_accuracy: 0.8170 - lr: 0.0050 Epoch 9/20 93/93 [==============================] - 217s 2s/step - loss: 0.6582 - accuracy: 0.8367 - val_loss: 0.6894 - val_accuracy: 0.8175 - lr: 0.0045 Epoch 10/20 93/93 [==============================] - 215s 2s/step - loss: 0.6550 - accuracy: 0.8367 - val_loss: 0.6754 - val_accuracy: 0.8156 - lr: 0.0041 Epoch 11/20 93/93 [==============================] - 214s 2s/step - loss: 0.6352 - accuracy: 0.8390 - val_loss: 0.6763 - val_accuracy: 0.8116 - lr: 0.0037 Epoch 12/20 93/93 [==============================] - 214s 2s/step - loss: 0.6317 - accuracy: 0.8364 - val_loss: 0.6641 - val_accuracy: 0.8174 - lr: 0.0033 Epoch 13/20 93/93 [==============================] - 214s 2s/step - loss: 0.6189 - accuracy: 0.8373 - val_loss: 0.6780 - val_accuracy: 0.8177 - lr: 0.0030 Epoch 14/20 93/93 [==============================] - 221s 2s/step - loss: 0.5932 - accuracy: 0.8385 - val_loss: 0.6378 - val_accuracy: 0.8182 - lr: 0.0027 Epoch 15/20 93/93 [==============================] - 235s 3s/step - loss: 0.5848 - accuracy: 0.8368 - val_loss: 0.6291 - val_accuracy: 0.8154 - lr: 0.0025 Epoch 16/20 93/93 [==============================] - 246s 3s/step - loss: 0.5683 - accuracy: 0.8385 - val_loss: 0.6319 - val_accuracy: 0.8126 - lr: 0.0022 Epoch 17/20 93/93 [==============================] - 252s 3s/step - loss: 0.5572 - accuracy: 0.8379 - val_loss: 0.6044 - val_accuracy: 0.8129 - lr: 0.0020 Epoch 18/20 93/93 [==============================] - 235s 3s/step - loss: 0.5448 - accuracy: 0.8379 - val_loss: 0.6075 - val_accuracy: 0.8118 - lr: 0.0018 Epoch 19/20 93/93 [==============================] - 242s 3s/step - loss: 0.5423 - accuracy: 0.8372 - val_loss: 0.5905 - val_accuracy: 0.8154 - lr: 0.0017 Epoch 20/20 93/93 [==============================] - 244s 3s/step - loss: 0.5379 - accuracy: 0.8377 - val_loss: 0.6036 - val_accuracy: 0.8140 - lr: 0.0015
Plot¶
In [ ]:
print(history.history.keys())
# summarize history for accuracy
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('Loss function - Resize dice')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'val'], loc='upper left')
plt.show()
dict_keys(['loss', 'accuracy', 'val_loss', 'val_accuracy', 'lr'])
Test¶
In [ ]:
x_test, y_test = test_generator[0]
for i in range(1, 5):
print(i)
x_temp, y_temp = test_generator[i]
x_test, y_test = np.vstack((x_test,x_temp)), np.vstack((y_test, y_temp))
print(x_test.shape)
print(y_test.shape)
pred = model.predict(x_test)
1 2 3 4 (160, 224, 224, 3) (160, 224, 224, 5) 5/5 [==============================] - 5s 953ms/step
Dice coefficient¶
In [ ]:
print("Dice coefficient: \t" + '{:.4}'.format(dice_coef(y_test, pred)))
print("Dice loss function: \t" + '{:.4}'.format(dice_loss_function(y_test, pred)))
print(dice_coef(y_test, y_test))
def dice_coef2(y, pred):
coef = 0
for i in range(y.shape[3]):
intersection = np.sum(y[:,:,:,i] * pred[:,:,:,i])
set_size = np.sum(y[:,:,:,i]) + np.sum(pred[:,:,:,i])
coef += (2*intersection)/set_size
return coef/5.0
print(dice_coef2(y_test, pred))
print(dice_loss_function(tf.constant(y_test), tf.constant(pred)))
Dice coefficient: 0.7449 Dice loss function: 0.7789 tf.Tensor(1.0, shape=(), dtype=float64) 0.22109141016943923 tf.Tensor(0.7789085055464489, shape=(), dtype=float64)
IoU¶
In [ ]:
class_names = ["background", "rigid_plastic", "cardboard", "metal", "soft_plastic"]
def mean_intersection_over_union(y, pred):
result = 0
for i in range(num_classes):
intersection = np.sum(y[:,:,:,i]*pred[:,:,:,i])
union = np.sum(y[:,:,:,i]) + np.sum(pred[:,:,:,i]) - intersection
print("IoU of " + class_names[i] + " is " + '{:.4}'.format(100*intersection/union))
result += intersection/union
return result/num_classes
print("mean intersection over union is " + '{:.4}'.format(100*mean_intersection_over_union(y_test, pred)))
IoU of background is 76.42 IoU of rigid_plastic is 0.728 IoU of cardboard is 8.312 IoU of metal is 0.09754 IoU of soft_plastic is 3.585 mean intersection over union is 17.83
Percentage of pixels categorized correctly¶
In [ ]:
# Calculation of the percentage of pixels that are categorized correctly
y_pred_binary = np.argmax(pred, axis=3).flatten()
correct_predictions2 = np.sum(np.equal(y_pred_binary, np.argmax(y_test, axis=3).flatten()))
n, h, w, _ = y_test.shape
pixels = n*h*w
accuracy = correct_predictions2 / pixels * 100.0
print("acuracy is " + '{:.4}'.format(accuracy))
acuracy is 78.93
Confusion Matrix¶
In [ ]:
from sklearn.metrics import confusion_matrix
#See lab 4 for explanation of Confusion Matrix
pred_flat = np.array([])
image_labels_flat = np.array([])
print(pred.shape)
for i in range(pred.shape[0]):
image_predictions = np.argmax(pred[i], axis=2).flatten()
pred_flat = np.hstack([pred_flat,image_predictions])
img_labels = np.argmax(y_test[i], axis=2).flatten()
image_labels_flat = np.hstack([image_labels_flat, img_labels])
image_labels_flat = np.hstack([image_labels_flat, np.arange(5)])
pred_flat = np.hstack([pred_flat, np.arange(5)])
conf_matrix = confusion_matrix(image_labels_flat, pred_flat)
print("1: background 2: rigid_plastic 3: cardboard 4: metal 5: soft_plastic\n")
print("Confusion Matrix")
print(conf_matrix-np.eye(5, dtype=np.int64))
np.set_printoptions(precision=2)
conf_matrix_norm = ((conf_matrix.T-np.eye(5, dtype=np.int64)) / np.sum(conf_matrix, axis=1)).T
print("\nNormalized Confusion Matrix")
print(np.round(conf_matrix_norm,3)*100)
(160, 224, 224, 5) 1: background 2: rigid_plastic 3: cardboard 4: metal 5: soft_plastic Confusion Matrix [[6336330 10 20 0 13] [ 64289 0 0 0 0] [1069063 2 22 0 1] [ 4418 0 0 0 0] [ 553988 0 4 0 0]] Normalized Confusion Matrix [[100. 0. 0. 0. 0.] [100. 0. 0. 0. 0.] [100. 0. 0. 0. 0.] [100. 0. 0. 0. 0.] [100. 0. 0. 0. 0.]]
Precision and Recall¶
In [ ]:
show_interesting_facts = False
if show_interesting_facts:
print("------------------------------ interesting facts ----------------------------------------")
print("percentage of pixels that are cardboard in training set: " + '{:.4}'.format(100*np.sum(conf_matrix[2,:])/np.sum(conf_matrix)))
print("percentage of pixels that are background in training set: " + '{:.4}'.format(100*np.sum(conf_matrix[0,:])/np.sum(conf_matrix)) + " and accuracy is " + '{:.4}'.format(accuracy))
print("Therefore the model is " + '{:.4}'.format(accuracy - 100*np.sum(conf_matrix[0,:])/np.sum(conf_matrix)) + " percentage points better in accuracy than always guessing background")
print("\npercentage of time the model guesses background: " + '{:.4}'.format(100*np.sum(conf_matrix[:,0])/np.sum(conf_matrix)))
print("percentage of time the model guesses cardboard: " + '{:.4}'.format(100*np.sum(conf_matrix[:,2])/np.sum(conf_matrix)))
print("\n----------------------------------------------------------------------------------------")
classes = ["background", "rigid plastic", "cardboard", "metal\t", "soft plastic"]
print("Precision with regards to each class:\n")
conf_matrix = conf_matrix - np.eye(5, dtype=np.int64)
for i in range(len(classes)):
if np.sum(conf_matrix[:,i]) == 0:
print( classes[i] + ": \t" + "no predictions")
else:
precision = 100*conf_matrix[i,i]/np.sum(conf_matrix[:,i])
print( classes[i] + ": \t" + '{:.4}'.format(precision))
# This is also just the diagonal of the normalized confusion matrix
print("\nRecall with regards to each class:\n")
for i in range(len(classes)):
if np.sum(conf_matrix[i,:]) == 0:
print(classes[i] + ": \t" + "no predictions")
else:
recall = 100*conf_matrix[i,i]/np.sum(conf_matrix[i,:])
print(classes[i] + ": \t" + '{:.4}'.format(recall))
Precision with regards to each class: background: 78.93 rigid plastic: 0.0 cardboard: 47.83 metal : no predictions soft plastic: 0.0 Recall with regards to each class: background: 100.0 rigid plastic: 0.0 cardboard: 0.002058 metal : 0.0 soft plastic: 0.0
Visual inspection¶
In [ ]:
for i in range(0, pred.shape[0], 10):
print("------------------------------------------------------------\nFor test imgages #" + str(i+1))
Make_Overlapping_Plot2(x_test[i].astype(np.uint8), y_test[i], pred=pred[i])
------------------------------------------------------------ For test imgages #1
------------------------------------------------------------ For test imgages #11
------------------------------------------------------------ For test imgages #21
------------------------------------------------------------ For test imgages #31
------------------------------------------------------------ For test imgages #41
------------------------------------------------------------ For test imgages #51
------------------------------------------------------------ For test imgages #61
------------------------------------------------------------ For test imgages #71
------------------------------------------------------------ For test imgages #81
------------------------------------------------------------ For test imgages #91
------------------------------------------------------------ For test imgages #101
------------------------------------------------------------ For test imgages #111
------------------------------------------------------------ For test imgages #121
------------------------------------------------------------ For test imgages #131
------------------------------------------------------------ For test imgages #141
------------------------------------------------------------ For test imgages #151
